{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from string import letters\n",
    "import numpy as np\n",
    "from pandas import DataFrame\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn import preprocessing\n",
    "from sklearn import cross_validation\n",
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.loadtxt(\"F:\\\\data\\\\lotteryodds\\\\train_dx_all.txt\",delimiter=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = data[:,0:39]\n",
    "y = data[:,39]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 15.  ,  14.  ,   5.  , ...,   1.2 ,   2.5 ,   2.5 ],\n",
       "       [ 20.  ,  15.  ,  19.  , ...,   1.05,   2.5 ,   2.5 ],\n",
       "       [ 15.  ,  14.  ,   6.  , ...,   1.05,   2.5 ,   2.5 ],\n",
       "       ..., \n",
       "       [ 15.  ,   3.  ,  16.  , ...,   0.53,   2.5 ,   2.5 ],\n",
       "       [ 20.  ,  12.  ,   8.  , ...,   1.  ,   2.5 ,   2.5 ],\n",
       "       [ 17.  ,  15.  ,  25.  , ...,   0.7 ,   2.5 ,   2.5 ]])"
      ]
     },
     "execution_count": 5,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  0.,  0., ...,  0.,  0.,  1.])"
      ]
     },
     "execution_count": 6,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26007L, 47L)"
      ]
     },
     "execution_count": 70,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "sel = VarianceThreshold(threshold=(.8 * (1 - .8)))\n",
    "X_New=sel.fit_transform(X)\n",
    "X_New.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_New' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-15-7fb7ba39dbe6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpreprocessing\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[1;31m# normalize the data attributes\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 3\u001b[1;33m \u001b[0mnormalized_X\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpreprocessing\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnormalize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_New\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      4\u001b[0m \u001b[1;31m# standardize the data attributes\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mstandardized_X\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpreprocessing\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mscale\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_New\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'X_New' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "# normalize the data attributes\n",
    "normalized_X = preprocessing.normalize(X_New)\n",
    "# standardize the data attributes\n",
    "standardized_X = preprocessing.scale(X_New)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 15.  ,  14.  ,   5.  , ...,   0.62,   2.5 ,   2.5 ],\n",
       "       [ 20.  ,  15.  ,  19.  , ...,   0.7 ,   2.5 ,   2.5 ],\n",
       "       [ 15.  ,  14.  ,   6.  , ...,   0.7 ,   2.5 ,   2.5 ],\n",
       "       ..., \n",
       "       [ 15.  ,   3.  ,  16.  , ...,   1.38,   2.5 ,   2.5 ],\n",
       "       [ 20.  ,  12.  ,   8.  , ...,   0.73,   2.5 ,   2.5 ],\n",
       "       [ 17.  ,  15.  ,  25.  , ...,   1.05,   2.5 ,   2.5 ]])"
      ]
     },
     "execution_count": 14,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "#from sklearn.ensemble import ExtraTreesClassifier\n",
    "#from sklearn.svm import LinearSVC\n",
    "#clf = ExtraTreesClassifier()\n",
    "#X_new = clf.fit(X_New, y).transform(X_New)\n",
    "from sklearn import preprocessing\n",
    "X_new = preprocessing.scale(X)\n",
    "\n",
    "#X_new = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit_transform(X_new, y)\n",
    "X_new\n",
    "\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import chi2\n",
    "X_new = SelectKBest(chi2, k=20).fit_transform(X, y)\n",
    "X_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10L, 1L)"
      ]
     },
     "execution_count": 62,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "test = np.loadtxt(\"F:\\\\data\\\\test_all_binary.txt\",delimiter=\"\\t\")\n",
    "test_X = test[:,0:1]\n",
    "test_y = test[:,1]\n",
    "test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(26007L, 1L)"
      ]
     },
     "execution_count": 61,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "X_new.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.53429362880886422"
      ]
     },
     "execution_count": 15,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(X_new, y)\n",
    "from sklearn import metrics\n",
    "predicted = cross_validation.cross_val_predict(clf,X_new,\n",
    "                                          y)\n",
    "metrics.accuracy_score(y, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'ExtraTreesClassifier' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-47-b6b5826ef49a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mmetrics\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mExtraTreesClassifier\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_new\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m# display the relative importance of each attribute\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfeature_importances_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'ExtraTreesClassifier' is not defined"
     ]
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "model = ExtraTreesClassifier()\n",
    "model.fit(X_new, y)\n",
    "# display the relative importance of each attribute\n",
    "print(model.feature_importances_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0.106, 0)]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.cross_validation import cross_val_score, ShuffleSplit\n",
    "names = range(3)\n",
    "\n",
    "rf = RandomForestRegressor(n_estimators=20, max_depth=4)\n",
    "scores = []\n",
    "for i in range(X_new.shape[1]):\n",
    "     score = cross_val_score(rf, X_new[:, i:i+1], y, scoring=\"r2\",\n",
    "                              cv=ShuffleSplit(len(X_new), 3, .3))\n",
    "     scores.append((round(np.mean(score), 3), names[i]))\n",
    "print sorted(scores, reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.59768523858961053"
      ]
     },
     "execution_count": 19,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn import cross_validation\n",
    "pca = PCA(n_components=20)\n",
    "x_pca = pca.fit_transform(X,y)\n",
    "clf = RandomForestClassifier()\n",
    "clf.fit(x_pca, y)\n",
    "from sklearn import metrics\n",
    "predicted = cross_validation.cross_val_predict(clf,x_pca,\n",
    "                                          y)\n",
    "metrics.accuracy_score(y, predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}